/*
 * @Version: 0.0.1
 * @Author: ider
 * @Date: 2021-08-04 14:09:58
 * @LastEditors: ider
 * @LastEditTime: 2021-08-04 16:25:12
 * @Description: get all cpu links
 */

const Apify = require('apify')
const cheerio = require('cheerio')
const nanoid = require('nanoid')
const crypto = require('crypto')
const {log} = Apify.utils

log.setLevel(log.LEVELS.INFO)

const getHash = (name)=> crypto.createHash('md5').update(name).digest('hex')



Apify.main(async () => {
    const proxyConfiguration = await Apify.createProxyConfiguration({ proxyUrls:['http://192.168.1.229:6600'] });
    const requestQueueCpuIndex = await Apify.openRequestQueue('cpu-index')

    const cpuIndexStore = await Apify.openKeyValueStore('cpu_index')
    const requestQueueCpuDetail = await Apify.openRequestQueue('cpu-detail')

    for (let year=2000;year<=2021;year++ ){
        let url = `https://www.techpowerup.com/cpu-specs/?mfgr=AMD&released=${year}&sort=name`
        requestQueueCpuIndex.addRequest({
            url,
        })
    }
    
    for (let year=2000;year<=2021;year++ ){
        if([2011,2012,2013].includes(year)){
            continue
        }
        let url = `https://www.techpowerup.com/cpu-specs/?mfgr=Intel&released=${year}&sort=name`
        requestQueueCpuIndex.addRequest({
            url,
        })
    }

        
    let url = `https://www.techpowerup.com/cpu-specs/?mfgr=Intel&released=2012&process=22%20nm&sort=name`
    requestQueueCpuIndex.addRequest({
        url,
    })
    
    url = `https://www.techpowerup.com/cpu-specs/?mfgr=Intel&released=2012&process=32%20nm&sort=name`
    requestQueueCpuIndex.addRequest({
        url,
    })


    const rootCrawler = new Apify.PlaywrightCrawler({
        requestQueue: requestQueueCpuIndex,
        maxConcurrency: 1,
        maxRequestRetries: 1000,
        proxyConfiguration,
        handlePageFunction: async ({
            page,
            request
        }) => {
            log.info('start')
            await Apify.utils.sleep(10 * 1000);
            let html = await page.content()
            let $ = cheerio.load(html)
            let trs = $('.processors ').find("tr")
            for (let tr of trs.toArray()) {
                let td = $(tr).find('td')

                if (td.length === 9) {
                    let Name = $(td[0]).text().trim()
                    let Codename = $(td[1]).text().trim()
                    let Cores = $(td[2]).text().trim()
                    let Clock = $(td[3]).text().trim()
                    let Socket = $(td[4]).text().trim()
                    let Process = $(td[5]).text().trim()
                    let L3_Cache = $(td[6]).text().trim()
                    let TDP = $(td[7]).text().trim()
                    let Released = $(td[8]).text().trim()
                    let Url = $(td[0]).find("a").attr('href')

                    let indexData = {
                        Name,Codename,Cores,Clock,Socket,Process,L3_Cache,TDP,Released,Url
                    }
                    // log.info(getHash(Url))
                    console.log(indexData)
                    console.log("uuid:",getHash(Url),indexData)
                    await cpuIndexStore.setValue(getHash(Url), indexData)
                    requestQueueCpuDetail.addRequest({
                        url:`https://www.techpowerup.com${Url}`,
                        userData:indexData
                    })
                }

                
            }
        },
        handleFailedRequestFunction: async ({
            request
        }) => {
            // This function is called when the crawling of a request failed too many times
            await Apify.pushData({
                url: request.url,
                succeeded: false,
                errors: request.errorMessages,
            })
        },
    })

    await rootCrawler.run()
})